/*
 * Copyright 2008 Carnegie Mellon University
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *  
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, 
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package edu.cmu.lti.sepia.task.evaluation;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import edu.cmu.lti.sepia.data.db.QAJudgmentResultDAO;
import edu.cmu.lti.sepia.data.db.QAJudgmentUnitDAO;
import edu.cmu.lti.sepia.data.db.SQL;
import edu.cmu.lti.sepia.data.db.TopicDAO;
import edu.cmu.lti.sepia.infra.eval.model.QAJudgmentResult;
import edu.cmu.lti.sepia.infra.eval.model.QAJudgmentUnit;
import edu.cmu.lti.sepia.infra.run.RunXmlFacade;
import edu.cmu.lti.sepia.infra.run.model.AnswerCandidate;
import edu.cmu.lti.sepia.infra.run.model.SelectionResult;
import edu.cmu.lti.sepia.infra.run.model.SelectionResultSet;
import edu.cmu.lti.sepia.infra.topic.model.Nugget;
import edu.cmu.lti.sepia.infra.topic.model.Topic;
import edu.cmu.lti.sepia.util.FormatUtil;
import edu.cmu.lti.sepia.util.IOUtil;

public class CCLQAEvaluationRunner {

  // output dir
  // submitted file
  
  public void calculate( SQL sql, String table, String trgLang, File runFile, File outputDir ) 
  throws SQLException, FileNotFoundException, IOException, Exception {
    
    System.out.println("--\nStarted scoring: "+runFile);
    
    // Topics FROM DB!!!
    System.out.print("Loading topics ...");
    List<Topic> topics = TopicDAO.getGoldStandard( sql, table, -1, null );
    TopicDAO.removeSimilarNuggets(topics);
    TopicDAO.generateExternalIds(topics,trgLang,4);
    TopicDAO.removeUnselected(topics);
    System.out.println("Done.");
    
    System.out.print("Calculating allowance ... ");
    Map<String,Double> allowanceMap = getAllowanceMap(topics);
    System.out.println("Done. "+allowanceMap);
    
    // System responses
    System.out.print("Parsing run result.");
    InputStream is = new FileInputStream(runFile);
    SelectionResultSet systemResponses = RunXmlFacade.deserializeSelectionResultSet(is);
    System.out.println("Done.");
    
    StringBuilder humanBrief = new StringBuilder();
    StringBuilder humanDetailed = new StringBuilder();
    StringBuilder autoDetailed = new StringBuilder();
    
    double f3SumHuman = 0;
    double f3SumAuto = 0;
    
    for ( Topic t : topics ) {
      System.out.println(t.getExternalId()+" "+t.getTitle());
      // Judgments from DB!!!
      List<QAJudgmentUnit> units = QAJudgmentUnitDAO.findJudgmentUnitsByTopic(sql, table, t.getExternalId());      
      List<QAJudgmentResult> results = QAJudgmentResultDAO.findResultsByTopic(sql, table, t.getExternalId());
      Map<String,List<JudgedNugget>> judgmentResult = getJugmentResult( t, units, results ); // Nugget
      //Map<String,List<JudgedNugget>> autoJudgmentResult = getAutoJudgmentResult(); 
      
      SelectionResult sr = systemResponses.get(t.getExternalId());
      if (sr==null) sr = new SelectionResult(); 
      
      double allowanceConstant = allowanceMap.get(t.getAnswerType());
      String runId = systemResponses.getMetadata().get("RUNID");
      List<JudgedNugget> perTopicJudgedNuggets = getPerTopicJudgedNuggets(t.getNuggets(), sr, judgmentResult);
      List<JudgedNugget> perTopicJudgedNuggetsAuto = getPerTopicJudgedNuggetsAuto(t.getNuggets(), sr, judgmentResult);
      
      Report rHuman = evaluatePerTopic(t, sr, judgmentResult, perTopicJudgedNuggets, allowanceConstant, runId, true);
      humanBrief.append(rHuman.brief);
      humanDetailed.append(rHuman.detailed);
      
      Report rAuto = evaluatePerTopic(t, sr, judgmentResult, perTopicJudgedNuggetsAuto, allowanceConstant, runId, false);
      autoDetailed.append(rAuto.detailed);
      
      f3SumHuman += rHuman.f3;
      f3SumAuto  += rAuto.f3;
    }
    humanDetailed.append("---- Overall Summary ----\n\n");
    humanDetailed.append("Average nugget F3 score for the run "+systemResponses.getMetadata().get("RUNID")+":\n");
    humanDetailed.append( FormatUtil.nf4(f3SumHuman / (double)topics.size())+"\n" );

    autoDetailed.append("---- Overall Summary ----\n\n");
    autoDetailed.append("Average nugget F3 score for the run "+systemResponses.getMetadata().get("RUNID")+":\n");
    autoDetailed.append( FormatUtil.nf4(f3SumAuto / (double)topics.size())+"\n" );

    if (!outputDir.exists()) {
      outputDir.mkdirs();
    }
    
    IOUtil.writeToFile(humanBrief.toString(), new File(outputDir, runFile.getName().replaceAll("xml$", "brief.human.txt")));
    IOUtil.writeToFile(humanDetailed.toString(), new File(outputDir, runFile.getName().replaceAll("xml$", "detail.human.txt")));
    IOUtil.writeToFile(autoDetailed.toString(), new File(outputDir, runFile.getName().replaceAll("xml$", "detail.auto.txt")));
    
    System.out.println("Done scoring: "+runFile+".");
  }
  
  private Report evaluatePerTopic(Topic topic, 
          SelectionResult systemResponses, 
          Map<String,List<JudgedNugget>> judgmentResult, 
          List<JudgedNugget> perTopicJudgedNuggets,
          double allowanceConstant, String runId, boolean isHuman ) {
    StringBuilder brief = new StringBuilder();
    StringBuilder detailed = new StringBuilder();
    
    brief.append("Submission for question "+topic.getExternalId()+" with items numbered:\n");
    
    detailed.append("======================================================================\n");
    detailed.append("Topic: "+topic.getExternalId()+" ("+topic.getTitle()+")\n");
    detailed.append("Question: "+topic.getQuestionSrc()+" / "+topic.getQuestionTrg()+"\n");
    detailed.append("Answer Type: "+topic.getAnswerType()+"\n\n");

    detailed.append("System responses:\n");
    
    for ( int i=0; i<systemResponses.size(); i++ ) {
      AnswerCandidate ac = systemResponses.get(i);
      detailed.append( (i+1)+" "+ac.getText()+"\n" );
      brief.append(topic.getExternalId()+" "+runId+" "+(i+1)+" - "+ac.getText()+"\n");
    }
    detailed.append("\n");
    detailed.append("Weighted nuggets where those matched with system reponses are marked:\n");
    
    brief.append("\n");
    brief.append("Nuggets assigned to items for question "+topic.getExternalId()+":\n");
        
    int largeL = 0;
    for ( int i=0; i<systemResponses.size(); i++ ) {
      AnswerCandidate ac = systemResponses.get(i);
      List<JudgedNugget> judgedNuggets = judgmentResult.get(ac.getText());
      if (judgedNuggets==null) continue;
      for ( int j=0; j<judgedNuggets.size(); j++ ) {
        JudgedNugget jNugget = judgedNuggets.get(j);
        if (jNugget.isMatched) {
          brief.append(topic.getExternalId()+" "+runId+" "+(i+1)+" "+(j+1)+"\n");
        }
      }
      
      largeL += ac.getText().length();
    }

    double largeR = 0; // sum of nugget weights
    double smallR = 0; // sum of matched nugget weights
    double smallA = 0; // num of nuggets matched
    for (int i=0; i<perTopicJudgedNuggets.size(); i++) {
      JudgedNugget jn = perTopicJudgedNuggets.get(i);
      String label;
      if (isHuman) {
        label = jn.isMatched?"x":" ";
      } else {
        label = "["+(jn.matchScore>0?"x":" ")+" "+FormatUtil.nf4(jn.matchScore)+"]";
      }
      
      detailed.append(" "+label+" "+(i+1)+" "
              +FormatUtil.nf4(jn.nugget.getWeight())+" "+jn.nugget.getText()+"\n" );
      // eval
      largeR += jn.nugget.getWeight();
      if (isHuman) {
        if ( jn.isMatched ) {
          smallA += 1;
          smallR += jn.nugget.getWeight(); 
        }
      } else {
        if ( jn.matchScore > 0 ) {
          smallA += jn.matchScore;
          smallR += jn.nugget.getWeight() * jn.matchScore; // TODO: check if we multiply
        }
      }
    }
    double allowance = allowanceConstant * smallA;
    double recall = (largeR>0) ? smallR / largeR : 0;
    double precision = largeL<allowance ? 1D : (largeL>0?(allowance/largeL):0);
    double f3 = (recall>0 && precision>0) ? (10D * recall * precision / ( 9D * precision + recall )) : 0;
    
    detailed.append("\n");
    detailed.append("Scores:\n");
    detailed.append("C         = "+allowanceConstant+"\n");
    detailed.append("a         = "+(isHuman?Math.round(smallA):FormatUtil.nf4(smallA))+"\n");
    detailed.append("L         = "+largeL+"\n");
    detailed.append("allowance = "+allowance+"\n");
    detailed.append("r         = "+FormatUtil.nf4(smallR)+"\n");
    detailed.append("R         = "+FormatUtil.nf4(largeR)+"\n");
    detailed.append("Recall    = "+FormatUtil.nf4(recall)+"\n");
    detailed.append("Precision = "+FormatUtil.nf4(precision)+"\n");
    detailed.append("F3        = "+FormatUtil.nf4(f3)+"\n");
    detailed.append("\n\n\n");
    
    brief.append("\n\n\n");
    
    Report r = new Report();
    r.detailed = detailed.toString();
    r.brief = brief.toString();
    r.f3 = f3;
    return r;
  }
  
  private List<JudgedNugget> getPerTopicJudgedNuggets(
          List<Nugget> nuggets, SelectionResult systemResponses,
          Map<String,List<JudgedNugget>> judgmentResult ) {
    List<JudgedNugget> perTopicJudgedNuggets = new ArrayList<JudgedNugget>(nuggets.size());
    for ( Nugget n : nuggets ) {
      JudgedNugget jn = new JudgedNugget();
      jn.nugget = n;
      jn.isMatched = false;
      perTopicJudgedNuggets.add( jn );
    }
    for ( int i=0; i<systemResponses.size(); i++ ) {
      AnswerCandidate ac = systemResponses.get(i);
      List<JudgedNugget> judgedNuggets = judgmentResult.get(ac.getText());
      if (judgedNuggets==null) continue;
      for ( int j=0; j<judgedNuggets.size(); j++ ) {
        JudgedNugget jNugget = judgedNuggets.get(j);
        if (jNugget.isMatched) {
          perTopicJudgedNuggets.get(j).isMatched = true;
        }
      }
    }
    return perTopicJudgedNuggets;
  }
  
  private List<JudgedNugget> getPerTopicJudgedNuggetsAuto(
          List<Nugget> nuggets, SelectionResult systemResponses,
          Map<String,List<JudgedNugget>> judgmentResult ) {
    List<JudgedNugget> perTopicJudgedNuggets = new ArrayList<JudgedNugget>(nuggets.size());
    for ( Nugget n : nuggets ) {
      double maxScore = 0;
      for ( AnswerCandidate ac : systemResponses ) {
        double score = getSimilarity( n.getText(), ac.getText() );
        maxScore = Math.max(score, maxScore);
      }
      
      JudgedNugget jn = new JudgedNugget();
      jn.nugget = n;
      jn.isMatched = ( maxScore > 0.5D );
      //binarization
      maxScore = jn.isMatched?1:0;
      jn.matchScore = maxScore;
      perTopicJudgedNuggets.add(jn);
    }
    return perTopicJudgedNuggets;
  }
  
  private double getSimilarity( String nugget, String systemResponse ) {
    String[] nuggetTokens = extractChars(nugget);
    String[] systemResponseTokens = extractChars(systemResponse);
    double numMatched = countCommonTokenByExactMatch(systemResponseTokens, nuggetTokens);
    return numMatched / (double)nuggetTokens.length;
  }
  
  public String[] extractChars( String sentence ) {
    String[] words = new String[sentence.length()];
    for (int i=0; i<words.length; i++) {
      words[i] = new String(new char[]{sentence.charAt(i)});
    }
    return words;
  }
  
  public static double countCommonTokenByExactMatch( String[] sysTokens, String[] refTokens ) {
    int result = 0;
    //System.out.println(Arrays.toString(refTokens) +"<=>"+Arrays.toString(sysTokens));
    
    String[] refTokens2 = refTokens.clone();
    SYSTEM : for ( int i = 0; i<sysTokens.length; i++ ) {
      for ( int j = 0; j<refTokens2.length; j++ ) {
        if ( refTokens2[j].equals("-") ) continue;
        if ( sysTokens[i].equals(refTokens2[j]) ) {
          refTokens2[j] = "-";
          result++;
          continue SYSTEM;
        }
      }
    }
    
    return result;
  }
  
  private Map<String,List<JudgedNugget>> getJugmentResult(
          Topic topic, List<QAJudgmentUnit> units, List<QAJudgmentResult> results) {
    Map<String,List<JudgedNugget>> judgmentResult = new LinkedHashMap<String,List<JudgedNugget>>();
    
    for ( int i=0; i<units.size(); i++ ) {
      QAJudgmentUnit u = units.get(i);
      List<JudgedNugget> judgedNuggets = new ArrayList<JudgedNugget>();
      for ( Nugget n : topic.getNuggets() ) {
        JudgedNugget judged = new JudgedNugget();
        judged.nugget = n;
        judged.isMatched = false;
        judgedNuggets.add( judged );
      }
      for ( QAJudgmentResult result : results ) { 
        if ( u.getJudgmentId() == result.getJudgmentId() && result.getJudgment().equals("1") ) {
          for ( int j=0; j<judgedNuggets.size(); j++ ) {
            Nugget n = judgedNuggets.get(j).nugget;
            if (n.getNuggetId() == result.getNuggetId()) {
              
              judgedNuggets.get(j).isMatched = true;
            }
          }
        }
      }
      judgmentResult.put(u.getSystemResponse(), judgedNuggets);
    }
    return judgmentResult;
  }
  
  private Map<String,Double> getAllowanceMap( List<Topic> topics ) {
    Map<String,Integer> counts = new HashMap<String,Integer>();
    Map<String,Double> sumAvgLength = new HashMap<String,Double>();
    for ( Topic t : topics ) {
      double avgCharLen = 0;
      for ( Nugget n : t.getNuggets() ) {
        avgCharLen += n.getText().length();
      }
      avgCharLen /= (double) t.getNuggets().size();
      
      String key = t.getAnswerType();
      Double sumAvgLen = sumAvgLength.get(key);
      if (sumAvgLen==null) sumAvgLen=0D;
      sumAvgLen = sumAvgLen+avgCharLen;
      sumAvgLength.put(key, sumAvgLen);
      Integer sumCount = counts.get(key);
      if (sumCount==null) sumCount=0;
      sumCount++;
      counts.put(key, sumCount);
    }
    
    Map<String,Double> allowanceMap = new HashMap<String,Double>();
    for ( String key : counts.keySet() ) {
      double allowance = (double)sumAvgLength.get(key) / (double)counts.get(key);
      allowance = Math.round(allowance * 10D)/10D;
      allowanceMap.put(key, allowance);
    }
    return allowanceMap;
  }
  
  class Report {
    String brief;
    String detailed;
    double f3;
  }
  
  class JudgedNugget {
    Nugget nugget;
    boolean isMatched; // used for human eval
    double matchScore; // used for auto eval
  }
  
  public static void main(String[] args) throws Exception {
    SQL sql = new SQL();
    String base = "C:/Users/hideki/Desktop/aclia/cclqa evaluation scoring";
    Pattern p = Pattern.compile("-(JA|CS|CT)-0");
//    Pattern p = Pattern.compile("-(JA)-0");
//    Pattern p = Pattern.compile("-(CS|CT)-0");
    File runDir = new File(base+"/submitted formal run - collaboration");
    File outputDir = new File(base+"/score report - collaboration");
    CCLQAEvaluationRunner c = new CCLQAEvaluationRunner();
    for (File runFile : runDir.listFiles()) {
      Matcher m = p.matcher(runFile.getName());
      if (m.find()) {
        String trgLang = m.group(1);
        String table = trgLang.toLowerCase()+"_formal";
        c.calculate(sql, table, trgLang, runFile, outputDir);
      }
    }
    sql.close();
  }
}
